which versions installed?
library(sparklyr)
package ‘sparklyr’ was built under R version 3.5.2
spark_installed_versions()
Last Spark version Availability
spark_available_versions(show_hadoop = TRUE) %>% tail()
Install Specific spark version
library(sparklyr)
#spark_install(version = "2.4.0")
Connect to two spark versions
conf232 <- spark_config() # Load variable with spark_config()
conf232$`sparklyr.cores.local` <- 2
conf232$`sparklyr.shell.driver-memory` <- "4G"
conf232$spark.memory.fraction <- 0.5
conf240 <- spark_config() # Load variable with spark_config()
conf240$`sparklyr.cores.local` <- 2
conf240$`sparklyr.shell.driver-memory` <- "4G"
conf240$spark.memory.fraction <- 0.5
sc240 <- spark_connect(master = "local",
version = "2.4.0",
config = conf240
)
sc232 <- spark_connect(master = "local",
version = "2.3.2",
config = conf232
)
Re-using existing Spark connection to local
Only the first connection is done!
Copy data set in different nodes
LS0tCnRpdGxlOiAiU2V0dGluZyBtdWx0aXBsZSBTcGFyayBOb2RlcyAiCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCiMjIHdoaWNoIHZlcnNpb25zIGluc3RhbGxlZD8KYGBge3J9CmxpYnJhcnkoc3BhcmtseXIpCnNwYXJrX2luc3RhbGxlZF92ZXJzaW9ucygpCmBgYAoKIyMgTGFzdCBTcGFyayB2ZXJzaW9uIEF2YWlsYWJpbGl0eQpgYGB7cn0Kc3BhcmtfYXZhaWxhYmxlX3ZlcnNpb25zKHNob3dfaGFkb29wID0gVFJVRSkgJT4lIHRhaWwoKQpgYGAKCiMjIEluc3RhbGwgU3BlY2lmaWMgc3BhcmsgdmVyc2lvbgpgYGB7cn0KbGlicmFyeShzcGFya2x5cikKI3NwYXJrX2luc3RhbGwodmVyc2lvbiA9ICIyLjQuMCIpCmBgYAoKCiMjIENvbm5lY3QgdG8gdHdvIHNwYXJrIHZlcnNpb25zCgpgYGB7cn0KY29uZjIzMiA8LSBzcGFya19jb25maWcoKSAgICMgTG9hZCB2YXJpYWJsZSB3aXRoIHNwYXJrX2NvbmZpZygpCmNvbmYyMzIkYHNwYXJrbHlyLmNvcmVzLmxvY2FsYCA8LSAyCmNvbmYyMzIkYHNwYXJrbHlyLnNoZWxsLmRyaXZlci1tZW1vcnlgIDwtICI0RyIKY29uZjIzMiRzcGFyay5tZW1vcnkuZnJhY3Rpb24gPC0gMC41Cgpjb25mMjQwIDwtIHNwYXJrX2NvbmZpZygpICAgIyBMb2FkIHZhcmlhYmxlIHdpdGggc3BhcmtfY29uZmlnKCkKY29uZjI0MCRgc3BhcmtseXIuY29yZXMubG9jYWxgIDwtIDIKY29uZjI0MCRgc3BhcmtseXIuc2hlbGwuZHJpdmVyLW1lbW9yeWAgPC0gIjRHIgpjb25mMjQwJHNwYXJrLm1lbW9yeS5mcmFjdGlvbiA8LSAwLjUKCnNjMjQwIDwtIHNwYXJrX2Nvbm5lY3QobWFzdGVyID0gImxvY2FsIiwKICAgICAgICAgICAgICAgICAgICAgIHZlcnNpb24gPSAiMi40LjAiLAogICAgICAgICAgICAgICAgICAgICAgY29uZmlnID0gY29uZjI0MAogICAgICAgICAgICAgICAgICAgICkKc2MyMzIgPC0gc3BhcmtfY29ubmVjdChtYXN0ZXIgPSAibG9jYWwiLAogICAgICAgICAgICAgICAgICAgIHZlcnNpb24gPSAiMi4zLjIiLAogICAgICAgICAgICAgICAgICAgIGNvbmZpZyA9IGNvbmYyMzIKICAgICAgICAgICAgICAgICAgICApCgoKYGBgCgpPbmx5IHRoZSBmaXJzdCBjb25uZWN0aW9uIGlzIGRvbmUhCgojIyBDb3B5IGRhdGEgc2V0IGluIGRpZmZlcmVudCBub2RlcwoKYGBge3J9CnNjMTUyCmBgYAoK